{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import gc\n",
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import psutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "year = \"2017\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = list(filter(lambda x: year in x, os.listdir(\"./data/dblp/json/\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_nodes = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "div_gb_factor = 1024 ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "availablememory: 123.933384GB\n"
     ]
    }
   ],
   "source": [
    "pc_mem = psutil.virtual_memory()\n",
    "print(\"availablememory: %fGB\" % float(pc_mem.available/div_gb_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1/12]\n",
      "before extraction, availablememory: 123.933384GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 191696/191696 [00:15<00:00, 12543.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84.46764969825745\n",
      "after extraction, availablememory: 113.143513GB\n",
      "[2/12]\n",
      "before extraction, availablememory: 113.143513GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1515282/1515282 [00:16<00:00, 91987.15it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66.62278699874878\n",
      "after extraction, availablememory: 109.440441GB\n",
      "[3/12]\n",
      "before extraction, availablememory: 109.440441GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 212011/212011 [00:18<00:00, 11495.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122.12509846687317\n",
      "after extraction, availablememory: 107.892975GB\n",
      "[4/12]\n",
      "before extraction, availablememory: 107.892975GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1474809/1474809 [00:15<00:00, 96941.42it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72.05730152130127\n",
      "after extraction, availablememory: 107.417507GB\n",
      "[5/12]\n",
      "before extraction, availablememory: 107.417507GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1464836/1464836 [00:13<00:00, 110788.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73.33997845649719\n",
      "after extraction, availablememory: 107.420265GB\n",
      "[6/12]\n",
      "before extraction, availablememory: 107.420265GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 183540/183540 [00:18<00:00, 10039.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "136.72631406784058\n",
      "after extraction, availablememory: 103.244801GB\n",
      "[7/12]\n",
      "before extraction, availablememory: 103.244808GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1503815/1503815 [00:15<00:00, 94443.43it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89.58476281166077\n",
      "after extraction, availablememory: 105.344109GB\n",
      "[8/12]\n",
      "before extraction, availablememory: 105.344109GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 235582/235582 [00:19<00:00, 12105.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "174.2288625240326\n",
      "after extraction, availablememory: 90.176731GB\n",
      "[9/12]\n",
      "before extraction, availablememory: 90.176731GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 312559/312559 [00:24<00:00, 12842.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160.53574323654175\n",
      "after extraction, availablememory: 71.208168GB\n",
      "[10/12]\n",
      "before extraction, availablememory: 71.208168GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 205287/205287 [00:20<00:00, 10186.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "187.8566997051239\n",
      "after extraction, availablememory: 77.194592GB\n",
      "[11/12]\n",
      "before extraction, availablememory: 77.194592GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 225540/225540 [00:21<00:00, 10537.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "208.46772384643555\n",
      "after extraction, availablememory: 64.825996GB\n",
      "[12/12]\n",
      "before extraction, availablememory: 64.825996GB\n",
      "file content obtained\n",
      "convert str to json finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1491569/1491569 [00:19<00:00, 74732.28it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124.82607412338257\n",
      "after extraction, availablememory: 67.149971GB\n"
     ]
    }
   ],
   "source": [
    "for i,filename in enumerate(filenames):\n",
    "    print(\"[{}/{}]\".format(i+1, len(filenames)))\n",
    "    start_time = time.time()\n",
    "    pc_mem = psutil.virtual_memory()\n",
    "    print(\"before extraction, availablememory: %fGB\" % float(pc_mem.available/div_gb_factor))\n",
    "    f = codecs.open(\"./data/dblp/json/{}\".format(filename), \"r\")\n",
    "    content = f.read()\n",
    "    print(\"file content obtained\")\n",
    "    f.close()\n",
    "    content_json = json.loads(content)\n",
    "    print(\"convert str to json finished\")\n",
    "    del content\n",
    "    gc.collect()\n",
    "    gc.collect()\n",
    "    with tqdm(content_json, total= len(content_json)) as t:\n",
    "        for content in t:\n",
    "            authors = []\n",
    "            for item in content:\n",
    "                if item[0] == \"author\":\n",
    "                    authors.append(item[1])\n",
    "                elif item[0] == \"year\":\n",
    "                    year = item[1]\n",
    "                elif item[0] == \"title\":\n",
    "                    title = item[1]\n",
    "            try:\n",
    "                for author in authors:\n",
    "                    author_info = author_nodes.get(author, [])\n",
    "                    author_info.append({})\n",
    "                    author_info[-1][\"year\"] = year\n",
    "                    author_info[-1][\"title\"] = title\n",
    "                    author_nodes[author] = author_info\n",
    "                    del author\n",
    "                del year\n",
    "                del title\n",
    "            except Exception as e:\n",
    "                pass\n",
    "            del authors\n",
    "            del content\n",
    "        gc.collect()\n",
    "        gc.collect()\n",
    "    del content_json\n",
    "    gc.collect()\n",
    "    gc.collect()\n",
    "    print(time.time() - start_time)\n",
    "    pc_mem = psutil.virtual_memory()\n",
    "    print(\"after extraction, availablememory: %fGB\" % float(pc_mem.available/div_gb_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"./data/dblp/article\", exist_ok= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = codecs.open(\"./data/dblp/article/{}.json\".format(\"-\".join(filename.split(\"-\")[:2])), \"w\")\n",
    "temp = json.dumps(author_nodes, ensure_ascii= False)\n",
    "f.write(temp)\n",
    "f.close()\n",
    "del temp\n",
    "gc.collect()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "availablememory: 62.181210GB\n"
     ]
    }
   ],
   "source": [
    "pc_mem = psutil.virtual_memory()\n",
    "print(\"availablememory: %fGB\" % float(pc_mem.available/div_gb_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "year2titles = {}\n",
    "for author, articles in author_nodes.items():\n",
    "    for info in articles:\n",
    "        try:\n",
    "            temp = year2titles.get(info[\"year\"], set())\n",
    "            temp.add(info[\"title\"])\n",
    "            year2titles[info[\"year\"]] = temp\n",
    "        except Exception as e:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for year, titles in year2titles.items():\n",
    "    year2titles[year] = list(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.makedirs(\"./data/dblp/year2articles\", exist_ok= True)\n",
    "f = codecs.open(\"./data/dblp/year2articles/{}.json\".format(\"-\".join(filename.split(\"-\")[:2])), \"w\")\n",
    "temp = json.dumps(year2titles, ensure_ascii= False)\n",
    "f.write(temp)\n",
    "del temp\n",
    "gc.collect()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "availablememory: 59.970711GB\n"
     ]
    }
   ],
   "source": [
    "pc_mem = psutil.virtual_memory()\n",
    "print(\"availablememory: %fGB\" % float(pc_mem.available/div_gb_factor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "year2info = {}\n",
    "for year in year2titles.keys():\n",
    "    word2count = {}\n",
    "    for title in year2titles[year]:\n",
    "        temp = title.split(\" \")\n",
    "        for word in temp:\n",
    "            c = word2count.get(word, 0)\n",
    "            c += 1\n",
    "            word2count[word] = c\n",
    "        del temp\n",
    "    year2info[year] = {}\n",
    "    year2info[year][\"titles\"] = year2titles[year]\n",
    "    year2info[year][\"top_words\"] = sorted(word2count.items(), key=lambda x: x[1], reverse= True)[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.makedirs(\"./data/dblp/year2info\", exist_ok= True)\n",
    "f = codecs.open(\"./data/dblp/year2info/{}.json\".format(\"-\".join(filename.split(\"-\")[:2])), \"w\")\n",
    "temp = json.dumps(year2info, ensure_ascii= False)\n",
    "f.write(temp)\n",
    "del temp\n",
    "gc.collect()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('of', 49802),\n",
       " ('for', 40680),\n",
       " ('and', 36200),\n",
       " ('in', 27338),\n",
       " ('the', 22367),\n",
       " ('A', 17490),\n",
       " ('on', 13548),\n",
       " ('a', 13434),\n",
       " ('with', 13178),\n",
       " ('to', 9874),\n",
       " ('The', 5183),\n",
       " ('using', 5182),\n",
       " ('An', 4970),\n",
       " ('Using', 4457),\n",
       " ('based', 4353),\n",
       " ('by', 3670),\n",
       " ('On', 3311),\n",
       " ('Based', 3287),\n",
       " ('from', 3255),\n",
       " ('an', 3116),\n",
       " ('Networks.', 2852),\n",
       " ('Analysis', 2686),\n",
       " ('With', 2564),\n",
       " ('analysis', 2258),\n",
       " ('Data', 2171),\n",
       " ('model', 2086),\n",
       " ('Learning', 2013),\n",
       " ('Wireless', 1963),\n",
       " ('via', 1900),\n",
       " ('Control', 1867),\n",
       " ('data', 1851),\n",
       " ('networks.', 1831),\n",
       " ('method', 1830),\n",
       " ('-', 1797),\n",
       " ('algorithm', 1797),\n",
       " ('approach', 1778),\n",
       " ('Model', 1748),\n",
       " ('Design', 1698),\n",
       " ('Systems.', 1675),\n",
       " ('Sensor', 1603),\n",
       " ('System', 1601),\n",
       " ('Network', 1545),\n",
       " ('Efficient', 1540),\n",
       " ('control', 1528),\n",
       " ('Detection', 1487),\n",
       " ('Algorithm', 1469),\n",
       " ('system', 1421),\n",
       " ('Systems', 1404),\n",
       " ('systems.', 1399),\n",
       " ('systems', 1398)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "year2info['2015']['top_words']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
